﻿#include "spider.h"
#include <QtNetwork>
#include <QTextCodec>
#include <QUrl>
#include <QDebug>
#include <QNetworkProxy>
#include <QEventLoop>
#include <QRegExp>
#include <QFile>
#include <QTextStream>

//#pragma execution_character_set("utf-8")

Spider::Spider(QObject *parent) : QObject(parent)
{

    all = "";
    timer.setInterval(10000);
    timer.setSingleShot(true);

    manager = new QNetworkAccessManager(this);

    QNetworkProxy proxy;
    proxy.setType(QNetworkProxy::HttpProxy);
    proxy.setHostName("proxy.neusoft.com");
    proxy.setPort(8080);
    proxy.setUser("ch-ch");
    proxy.setPassword("UJMujm009");
    QNetworkProxy::setApplicationProxy(proxy);

    manager->setProxy(proxy);

    get(QUrl("https://github.com/ftylitak/qzxing/tree/master/src"));

    handleData();

    qDebug()<<"work finish.";
}

Spider::~Spider()
{

}

void Spider::replyFinished(QNetworkReply *reply)
{

//    QTextCodec *codec = QTextCodec::codecForLocale();
//    QString all = codec->toUnicode(reply->readAll());

//    reply->deleteLater();
}

void Spider::httpReadyRead()
{
    QTextCodec *codec = QTextCodec::codecForLocale();
    QString data = codec->toUnicode(reply->readAll());
    all = all + data;
}

void Spider::writeFile(QString path, QString data)
{
    qDebug()<<"file = "<< path;
    QFile file(path);

    if(!file.open(QIODevice::WriteOnly | QIODevice::Text)){
        qDebug()<<"write file error.";
        return;
    }

    QTextStream stream(&file);

    if(data.isEmpty())
        stream<<all;
    else
        stream<<data;

    stream.flush();

    file.close();
}

void Spider::get(QUrl url)
{
    QNetworkRequest request(url);

    request.setRawHeader("User-Agent", "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/68.0.3440.106 Safari/537.36");

    reply = manager->get(request);

    QEventLoop loop;
    connect(&timer, &QTimer::timeout, &loop, &QEventLoop::quit);
    connect(reply, &QNetworkReply::finished, &loop, &QEventLoop::quit);
    connect(reply, &QIODevice::readyRead, this, &Spider::httpReadyRead);

    timer.start();
    loop.exec();

    if(timer.isActive()){
        timer.stop();
        if(reply->error() != QNetworkReply::NoError){
            qDebug()<<"error "<<reply->errorString();
        }
        else{
            QVariant var = reply->attribute(QNetworkRequest::HttpStatusCodeAttribute);
            int nStatusCode = var.toInt();
            qDebug()<<"code: "<<nStatusCode;
        }
    }
    else{
        disconnect(reply, &QNetworkReply::finished, &loop, &QEventLoop::quit);
        disconnect(reply, &QNetworkReply::readyRead, this, &Spider::httpReadyRead);
        reply->abort();
        reply->deleteLater();
        qDebug()<<"timerout";
    }
}

void Spider::handleData()
{
    QList<dataType> list;

    if(!all.isEmpty()){
//        writeFile();  //test

        QRegExp rx("<a class=\"js-navigation-open\" title=\"(.*)\" id=(.*)href=\"(.*)\"");
        rx.setMinimal(true);
        int count = 0;
        int pos = 0;

        while((pos = rx.indexIn(all, pos)) != -1){
            ++count;
            pos += rx.matchedLength();

            dataType data = {rx.cap(1), "https://github.com" + rx.cap(3)};
            list.append(data);
        }
    }

    all = "";

    foreach (dataType data, list) {
        if(data.title.contains(".")){
            //is file
            qDebug()<<data.title<<data.href;
            get(QUrl(data.href));
            handleFileData(data.title, data.href);
        }
        else{
            //is dir
            get(QUrl(data.href));
            handleData();
        }
    }
}

void Spider::handleFileData(QString title, QString href)
{
    Q_UNUSED(title);

    if(!all.isEmpty()){

        QString data = "";

        QRegExp rx("class=\"blob-code blob-code-inner js-file-line\">(.*)</td>");
        rx.setMinimal(true);
        int count = 0;
        int pos = 0;

        while((pos = rx.indexIn(all, pos)) != -1){
            ++count;
            pos += rx.matchedLength();

            QString d = rx.cap(1);

//            qDebug()<<count<<d;

            QRegExp rx1("<span(.*)\">");
            rx1.setMinimal(true);

            d.replace(rx1, "");

            d.replace("</span>", "");
            d.replace("&quot;", "\"");
            d.replace("&lt;", "<");
            d.replace("&gt;", ">");
            d.replace("&amp;", "&");
            d.replace("&#39;", "'");

            if(d == "\n")
                data = data + d;
            else
                data = data + d + "\n";

        }

        //根据路径解析文件层级
        QFileInfo info(href.replace("https://github.com/", ""));

        QDir dir;
        QString path = info.absoluteDir().path();
        if(!dir.exists(path)){
            //创建目录
            bool r = dir.mkpath(path);
        }
        writeFile(info.absoluteFilePath(), data);
    }

    all = "";
}
